package scalapackage.testspark

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

/**
  * Created by Germmy on 2018/5/3.
  * 今天第一天跑，没跑起来，报啥winutils错误，貌似是windows电脑没有把环境配好
  */
object SparkWC {

  def main(args: Array[String]) {
    val sparkConf: SparkConf = new SparkConf().setAppName("SparkWC")//.setMaster("local[*]")
    val sc: SparkContext = new SparkContext(sparkConf)
    val lines: RDD[String] = sc.textFile(args(0))
    val by: RDD[(String, Int)] = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _).sortBy(_._2)
//    val toBuffer: mutable.Buffer[(String, Int)] = by.collect().toBuffer
//    println(toBuffer)
    by.saveAsTextFile(args(1))
    sc.stop()
  }

}
